package com.robet.webmagic;

import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;

import com.robet.webmagic.processor.GithubRepoPageProcessor;
import com.robet.webmagic.processor.OschinaRepoPageProcessor;

import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.JsonFilePipeline;

@SpringBootApplication
public class MagicProjectRunner  implements CommandLineRunner {
	
	public static void main(String[] args) throws Exception {
		SpringApplication.run(MagicProjectRunner.class, args);
	}

	@Override
	public void run(String... arg0) throws Exception {
		Spider.create(new OschinaRepoPageProcessor()).
			addUrl("https://www.oschina.net/news/industry")
			.addPipeline(new JsonFilePipeline("/Users/windmap/work/tmp/webmagic/"))
			.thread(5).run();
	}
	
	
	
}
